package util;


import java.io.IOException;
import org.apache.http.client.params.CookiePolicy;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class SpiderBodyXMLSougou {
	//抓取失败后多次重启，每次重启间隔的增量
	final int errorSleepIncrement = 30000;
	HtmlPage spiderPage = null;
	WebClient webClient = null;
	String bodyString = null;

	public synchronized String  getWebSiteBodyString() {
		String bodyString = this.bodyString;
		//System.out.println(bodyString);
		return bodyString;
	}

	public void clear() {
		webClient.closeAllWindows();
		webClient.getCookieManager().clearCookies();
	}

	public void SpiderPageToXML(String site) {
		//这个循环的目的是在出现错误是，可以多次重复抓取
		for (int reConectTimes = 0; reConectTimes < repeatTimes;) {
			webClient = new WebClient(BrowserVersion.FIREFOX_3_6);
			System.setProperty("apache.commons.httpclient.cookiespec",
					CookiePolicy.BROWSER_COMPATIBILITY);
			webClient.setJavaScriptEnabled(false);
			try {
				//sougou是直接封，如果被封下一句就会出异常，跳到catch块
				this.spiderPage = webClient.getPage(site);
				this.bodyString = spiderPage.getBody().asXml();
				//抓取成功后直接退出循环
				reConectTimes = repeatTimes;
			} catch (IOException e) {
				// 出错链连接错误，连续几次，如果连续连接最多次数title写成0000、content写成0000，否则暂停10秒继续连接
				if (reConectTimes == repeatTimes - 1) {
					this.bodyString = "0";
				} else
					try {
						//出错后且没到错多出错次数，会延时一段继续执行
						System.out.println("now it will spider again");
						Thread.sleep(interval + errorSleepIncrement * reConectTimes);
					} catch (InterruptedException e1) {
						e1.printStackTrace();
					}
			}
			webClient.closeAllWindows();
			webClient.getCookieManager().clearCookies();
			reConectTimes++;
		}
	}

	int repeatTimes = 0;
	int interval = 0;
	public void setInterval(int interval) {
		this.interval = interval;
	}
	public void setRepeatTimes(int repeatTimes) {
		this.repeatTimes = repeatTimes;
	}
}
